{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "demo_refactored_dianping_classification_with_BERT_fastai.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wshuyi/demo_chinese_text_classification_bert_fastai/blob/master/demo_refactored_dianping_classification_with_BERT_fastai.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T-HkYxo2GZT2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fastai.text import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DUU3lFXrXAUw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://github.com/wshuyi/public_datasets/raw/master/dianping.csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nDfYplnJM0A-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv(\"dianping.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z8m0iT14NAJz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUWWbsurNHRh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, test = train_test_split(df, test_size=.2, random_state=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTlBj0_vNOGs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, valid = train_test_split(train, test_size=.2, random_state=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5M58ohRXNRma",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwYryualNV1U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(valid)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iWWYv75WNW9l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ot3umGoNSw2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3gQIfrZJ8h7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install pytorch-transformers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DGPWStCvKASr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pytorch_transformers import BertTokenizer, BertForSequenceClassification"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dS1_jNESiEv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_model = \"bert-base-chinese\"\n",
        "max_seq_len = 128\n",
        "batch_size = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JiKPDWViStZS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_tokenizer = BertTokenizer.from_pretrained(bert_model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LxXk9yB1V0nB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "list(bert_tokenizer.vocab.items())[2000:2005]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NYIrjtkHW2F2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_vocab = Vocab(list(bert_tokenizer.vocab.keys()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aHXvwk8VGyq1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BertFastaiTokenizer(BaseTokenizer):\n",
        "    def __init__(self, tokenizer, max_seq_len=128, **kwargs):\n",
        "        self.pretrained_tokenizer = tokenizer\n",
        "        self.max_seq_len = max_seq_len\n",
        "\n",
        "    def __call__(self, *args, **kwargs):\n",
        "        return self\n",
        "\n",
        "    def tokenizer(self, t):\n",
        "        return [\"[CLS]\"] + self.pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BW8yrL48W7iX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tok_func = BertFastaiTokenizer(bert_tokenizer, max_seq_len=max_seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0HD1qCGCVMMe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_fastai_tokenizer = Tokenizer(\n",
        "    tok_func=tok_func,\n",
        "    pre_rules = [],\n",
        "    post_rules = []\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTOtOYvvNfVh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path = Path(\".\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FmN5wWFNbcj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "databunch = TextClasDataBunch.from_df(path, train, valid, test,\n",
        "                  tokenizer=bert_fastai_tokenizer,\n",
        "                  vocab=bert_vocab,\n",
        "                  include_bos=False,\n",
        "                  include_eos=False,\n",
        "                  text_cols=\"comment\",\n",
        "                  label_cols='sentiment',\n",
        "                  bs=batch_size,\n",
        "                  collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n",
        "             )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lAH_8yNQOEI7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "databunch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g5wzcUQ1YJ7y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyNoTupleModel(BertForSequenceClassification):\n",
        "  def forward(self, *args, **kwargs):\n",
        "    return super().forward(*args, **kwargs)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8bWsk4psVJd2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_pretrained_model = MyNoTupleModel.from_pretrained(bert_model, num_labels=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBM9tcDdVb8P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_func = nn.CrossEntropyLoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zoZZjQ9pPbdt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(databunch, \n",
        "                bert_pretrained_model,\n",
        "                loss_func=loss_func,\n",
        "                metrics=accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HU-sNOHfTpTp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.lr_find()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPFIFxcnUB9P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.recorder.plot()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDJDRZRsRGFI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.fit_one_cycle(2, 2e-5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHNMEpOAH3yN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dumb_series_prediction(n):\n",
        "  preds = []\n",
        "  for loc in range(n):\n",
        "    preds.append(int(learn.predict(test.iloc[loc]['comment'])[1]))\n",
        "  return preds"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NuhWUTdhn_UF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = dumb_series_prediction(len(test))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PvPRjZ_xoB80",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds[:10]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9NCp8k5hcnd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcJbaF-7hezH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(classification_report(test.sentiment, preds))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HWZXJ3iw-Zdv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(confusion_matrix(test.sentiment, preds))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4IKLzdgDp2j8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}